Rewrite dependency installation in Python
authorAlex Crichton <alex@alexcrichton.com>
Tue, 7 Jul 2015 20:55:36 +0000 (13:55 -0700)
committerAlex Crichton <alex@alexcrichton.com>
Tue, 7 Jul 2015 22:16:09 +0000 (15:16 -0700)
This commit aims to have the end goal of adding AppVeyor CI support to this
repo, and along the way it ended up meaning that the dependency installation
bits were rewritten in Python. This has a number of benefits:

* Python is more portable than shell
* Python is more readable than shell
* curl is no longer required on Windows (powershell is used for downloads)

There are also a few minor updates made as part of this commit as well:

* The README has been updated in how to build Cargo
* We now use `sudo: false` on Travis for faster builds. This is done by
  specifying packages to install instead of installing them ourselves.
* pkg-config is no longer listed as a required program

.travis.install.deps.sh
.travis.yml
README.md
appveyor.yml [new file with mode: 0644]
configure
src/etc/dl-snapshot.py
src/etc/download.py [new file with mode: 0644]
src/etc/install-deps.py [new file with mode: 0644]

index 8e1eb000892b05c02e1e82c71716b860f4753103..387cf1dd0cfb74dd5922300d0367a9a2a9328e36 100755 (executable)
@@ -1,75 +1,5 @@
-#!/bin/bash
+#!/bin/sh
 
-set -x
-set -e
+set -ex
 
-if [ "${TRAVIS_OS_NAME}" = "osx" ] || [ "${PLATFORM}" = "mac" ] || [ "`uname`" = "Darwin" ]; then
-    target=apple-darwin
-elif [ "${OS}" = "Windows_NT" ] || [ "${PLATFORM}" = "win" ]; then
-    windows=1
-else
-    target=unknown-linux-gnu
-fi
-
-if [ "${TRAVIS}" = "true" ] && [ "${target}" = "unknown-linux-gnu" ]; then
-    # Install a 32-bit compiler for linux
-    sudo apt-get update
-    if [ "${BITS}" = "32" ]; then
-        sudo apt-get install libssl-dev:i386
-    fi
-    sudo apt-get install g++-multilib lib32stdc++6
-fi
-
-url=https://static.rust-lang.org/dist/`cat src/rustversion.txt`
-
-# On unix hosts install both 32 and 64-bit libraries, but respect BITS to
-# determine what arch should be used by default. On windows we don't use 32/64
-# libraries, but instead we install msvc as an alternate architecture.
-if [ -z "${windows}" ]; then
-    if [ "${BITS}" = "32" ]; then
-        cargo_extra=x86_64-$target
-        cargo_host=i686-$target
-    else
-        cargo_extra=i686-$target
-        cargo_host=x86_64-$target
-    fi
-    libdir=lib
-else
-    if [ "${BITS}" = "32" ]; then
-        cargo_host=i686-pc-windows-gnu
-    elif [ "${MSVC}" = "1" ]; then
-        cargo_host=x86_64-pc-windows-msvc
-    else
-        cargo_host=x86_64-pc-windows-gnu
-        cargo_extra=x86_64-pc-windows-msvc
-    fi
-    libdir=bin
-fi
-
-rm -rf rustc *.tar.gz
-curl -O $url/rustc-nightly-$cargo_host.tar.gz
-tar xfz rustc-nightly-$cargo_host.tar.gz
-
-if [ ! -z "${cargo_extra}" ]; then
-    curl -O $url/rustc-nightly-$cargo_extra.tar.gz
-    tar xfz rustc-nightly-$cargo_extra.tar.gz
-
-    cp -r rustc-nightly-$cargo_extra/rustc/$libdir/rustlib/$cargo_extra \
-          rustc-nightly-$cargo_host/rustc/$libdir/rustlib
-    cp -r rustc-nightly-$cargo_extra/rustc/$libdir/rustlib/$cargo_extra/bin \
-          rustc-nightly-$cargo_host/rustc/$libdir/rustlib/$cargo_host
-    (cd rustc-nightly-$cargo_host && \
-     find rustc/$libdir/rustlib/$cargo_extra -type f | \
-     sed 's/^rustc\//file:/' >> rustc/manifest.in)
-    (cd rustc-nightly-$cargo_host && \
-     find rustc/$libdir/rustlib/$cargo_host/bin -type f | \
-     sed 's/^rustc\//file:/' >> rustc/manifest.in)
-    rm -rf rustc-nightly-$cargo_extra
-    rm -f rustc-nightly-$cargo_extra.tar.gz
-fi
-
-./rustc-nightly-$cargo_host/install.sh --prefix=rustc
-rm -rf rustc-nightly-$cargo_host
-rm -f rustc-nightly-$cargo_host.tar.gz
-
-set +x
+python src/etc/install-deps.py
index 1f0ceafaeb6679d91631f36c8185ded581875146..2dd3a38e77274e5353685adc06efda0a12196dc0 100644 (file)
@@ -1,25 +1,27 @@
 language: rust
+sudo: false
 install:
-  - sh ./.travis.install.deps.sh
+  - python src/etc/install-deps.py
 script:
-  - ./configure --local-rust-root=`pwd`/rustc
+  - ./configure --local-rust-root=`pwd`/rustc --prefix=$HOME/cargo-install
   - make
   - make test
   - make distcheck
   - make doc
-  - sudo make install
-  - sudo make uninstall
+  - make install
+  - make uninstall
 after_success: |
   [ $TRAVIS_BRANCH = master ] &&
   [ $TRAVIS_PULL_REQUEST = false ] &&
   [ $(uname -s) = Linux ] &&
-  sudo pip install ghp-import &&
-  ghp-import -n target/doc &&
-  git push -f https://${TOKEN}@github.com/${TRAVIS_REPO_SLUG}.git gh-pages
+  pip install ghp-import --user $USER &&
+  $HOME/.local/bin/ghp-import -n target/doc &&
+  git push -qf https://${TOKEN}@github.com/${TRAVIS_REPO_SLUG}.git gh-pages
 env:
   global:
     - secure: scGpeetUfba5RWyuS4yt10bPoFAI9wpHEReIFqEx7eH5vr2Anajk6+70jW6GdrWVdUvdINiArlQ3An2DeB9vEUWcBjw8WvuPtOH0tDMoSsuVloPlFD8yn1Ac0Bx9getAO5ofxqtoNg+OV4MDVuGabEesqAOWqURNrBC7XK+ntC8=
     - RUST_TEST_THREADS=1
+
 os:
   - linux
   - osx
@@ -27,3 +29,9 @@ os:
 branches:
   only:
     - master
+
+addons:
+  apt:
+    packages:
+    - g++-multilib
+    - lib32stdc++6
index 25309ba1358fd830677183030b0675c1de4f7f24..20786fcc1b8cdb8196f5a664b5a8e007abf2703d 100644 (file)
--- a/README.md
+++ b/README.md
@@ -1,21 +1,16 @@
 Cargo downloads your Rust project’s dependencies and compiles your project.
 
-Learn more at http://doc.crates.io/.
+Learn more at http://doc.crates.io/
 
-## Installing cargo from nightlies
+## Installing Cargo
 
-Cargo has nightlies available for use. The cargo source is not always guaranteed
-to compile on rust master as it may lag behind by a day or two. Nightlies,
-however, will run regardless of this fact!
+Cargo is distributed by default with Rust, so if you've got `rustc` installed
+locally you probably also have `cargo` installed locally.
 
-```sh
-triple=x86_64-unknown-linux-gnu
-curl -O https://static.rust-lang.org/cargo-dist/cargo-nightly-$triple.tar.gz
-tar xf cargo-nightly-$triple.tar.gz
-./cargo-nightly-$triple/install.sh
-```
-
-Nightlies are available for the following triples:
+If, however, you would like to install Cargo from the nightly binaries that are
+generated, you may also do so! Note that these nightlies are not official
+binaries, so they are only provided in one format with one installation method.
+Each tarball below contains a top-level `install.sh` script to install Cargo.
 
 * [`x86_64-unknown-linux-gnu`](https://static.rust-lang.org/cargo-dist/cargo-nightly-x86_64-unknown-linux-gnu.tar.gz)
 * [`i686-unknown-linux-gnu`](https://static.rust-lang.org/cargo-dist/cargo-nightly-i686-unknown-linux-gnu.tar.gz)
@@ -23,28 +18,27 @@ Nightlies are available for the following triples:
 * [`i686-apple-darwin`](https://static.rust-lang.org/cargo-dist/cargo-nightly-i686-apple-darwin.tar.gz)
 * [`x86_64-pc-windows-gnu`](https://static.rust-lang.org/cargo-dist/cargo-nightly-x86_64-pc-windows-gnu.tar.gz)
 * [`i686-pc-windows-gnu`](https://static.rust-lang.org/cargo-dist/cargo-nightly-i686-pc-windows-gnu.tar.gz)
+* [`x86_64-pc-windows-msvc`](https://static.rust-lang.org/cargo-dist/cargo-nightly-x86_64-pc-windows-msvc.tar.gz)
 
-Note that if you're using the windows snapshot you will need Mingw-w64 installed
-as well as MSYS. The installation script needs to be run inside the MSYS shell.
+Note that if you're on Windows you will have to run the `install.sh` script from
+inside an MSYS shell, likely from a MinGW-64 installation.
 
-## Compiling cargo
+## Compiling from Source
 
 Cargo requires the following tools and packages to build:
 
 * `rustc`
 * `python`
-* `curl`
+* `curl` (on Unix)
 * `cmake`
-* `pkg-config`
-* OpenSSL headers (`libssl-dev` package on ubuntu)
+* OpenSSL headers (only for Unix, this is the `libssl-dev` package on ubuntu)
 
 Cargo can then be compiled like many other standard unix-like projects:
 
 ```sh
 git clone https://github.com/rust-lang/cargo
 cd cargo
-git submodule update --init
-./.travis.install.deps.sh
+python src/etc/install-deps.py
 ./configure --local-rust-root="$PWD"/rustc
 make
 make install
@@ -60,7 +54,9 @@ $ ./configure --target=i686-unknown-linux-gnu,x86_64-unknown-linux-gnu
 
 ## Adding new subcommands to Cargo
 
-Cargo is designed to be extensible with new subcommands without having to modify Cargo itself. See [the Wiki page][third-party-subcommands] for more details and a list of known community-developed subcommands.
+Cargo is designed to be extensible with new subcommands without having to modify
+Cargo itself. See [the Wiki page][third-party-subcommands] for more details and
+a list of known community-developed subcommands.
 
 [third-party-subcommands]: https://github.com/rust-lang/cargo/wiki/Third-party-cargo-subcommands
 
diff --git a/appveyor.yml b/appveyor.yml
new file mode 100644 (file)
index 0000000..d1071f6
--- /dev/null
@@ -0,0 +1,19 @@
+environment:
+  matrix:
+  - MSVC: 1
+    BITS: 64
+    TARGET: x86_64-pc-windows-msvc
+
+install:
+  - python src/etc/install-deps.py
+  - python src/etc/dl-snapshot.py %TARGET%
+  - call "C:\Program Files (x86)\Microsoft Visual Studio 12.0\VC\vcvarsall.bat" amd64
+  - SET PATH=%PATH%;%cd%/rustc/bin
+  - SET PATH=%PATH%;%cd%/target/snapshot/cargo/bin
+  - rustc -V
+  - cargo -V
+
+build: false
+
+test_script:
+  - cargo test
index a7f9881f2a9ff929c7d217aba98eb6e10dd03763..e5590d923ed40b703959a8139505afaf3d2ab384 100755 (executable)
--- a/configure
+++ b/configure
@@ -261,10 +261,10 @@ need_cmd uname
 need_cmd date
 need_cmd tr
 need_cmd sed
-need_cmd file
 need_cmd cmake
-need_cmd pkg-config
-need_cmd curl
+if [ "${OS}" != "Windows_NT" ]; then
+    need_cmd curl
+fi
 
 CFG_SRC_DIR="$(cd $(dirname $0) && pwd)/"
 CFG_BUILD_DIR="$(pwd)/"
index a7f8c19070840d12813f0dc7dca1c5d4079c2f06..23f9c594929037c234c6a99b6a937d7bc95713f6 100644 (file)
@@ -1,12 +1,9 @@
-import distutils.spawn
+import download
 import hashlib
 import os
-import subprocess
-import sys
-import tarfile
-import shutil
-import contextlib
 import re
+import shutil
+import sys
 
 datere = re.compile('^\d{4}-\d{2}-\d{2}')
 cksumre = re.compile('^  ([^ ]+) ([^$]+)$')
@@ -91,21 +88,9 @@ if os.path.exists(dl_path):
         exists = True
 
 if not exists:
-    ret = subprocess.call(["curl", "-o", dl_path, url])
-    if ret != 0:
-        raise Exception("failed to fetch url")
+    download.get(url, dl_path)
     h = hashlib.sha1(open(dl_path, 'rb').read()).hexdigest()
     if h != hash:
         raise Exception("failed to verify the checksum of the snapshot")
 
-with contextlib.closing(tarfile.open(dl_path)) as tar:
-    for p in tar.getnames():
-        name = p.replace("cargo-nightly-" + triple + "/", "", 1)
-        fp = os.path.join(dst, name)
-        print("extracting " + p)
-        tar.extract(p, dst)
-        tp = os.path.join(dst, p)
-        if os.path.isdir(tp) and os.path.exists(fp):
-            continue
-        shutil.move(tp, fp)
-shutil.rmtree(os.path.join(dst, 'cargo-nightly-' + triple))
+download.unpack(dl_path, dst)
diff --git a/src/etc/download.py b/src/etc/download.py
new file mode 100644 (file)
index 0000000..27d4a86
--- /dev/null
@@ -0,0 +1,38 @@
+import contextlib
+import os
+import shutil
+import subprocess
+import sys
+import tarfile
+
+def get(url, path):
+    # see http://serverfault.com/questions/301128/how-to-download
+    if sys.platform == 'win32':
+        run(["PowerShell.exe", "/nologo", "-Command",
+             "(New-Object System.Net.WebClient).DownloadFile('" + url +
+                "', '" + path + "')"])
+    else:
+        run(["curl", "-o", path, url])
+
+def unpack(tarball, dst, quiet=False):
+    if quiet:
+        print("extracting " + tarball)
+    fname = os.path.basename(tarball).replace(".tar.gz", "")
+    with contextlib.closing(tarfile.open(tarball)) as tar:
+        for p in tar.getnames():
+            name = p.replace(fname + "/", "", 1)
+            fp = os.path.join(dst, name)
+            if not quiet:
+                print("extracting " + p)
+            tar.extract(p, dst)
+            tp = os.path.join(dst, p)
+            if os.path.isdir(tp) and os.path.exists(fp):
+                continue
+            shutil.move(tp, fp)
+    shutil.rmtree(os.path.join(dst, fname))
+
+def run(args):
+    print("running: " + ' '.join(args))
+    ret = subprocess.call(args)
+    if ret != 0:
+        raise Exception("failed to fetch url: " + url)
diff --git a/src/etc/install-deps.py b/src/etc/install-deps.py
new file mode 100644 (file)
index 0000000..c5be017
--- /dev/null
@@ -0,0 +1,73 @@
+#!/usr/bin/env python
+
+import contextlib
+import download
+import os
+import shutil
+import sys
+import tarfile
+
+if os.environ.get('BITS') == '32':
+    host_bits = 'i686'
+    extra_bits = 'x86_64'
+else:
+    host_bits = 'x86_64'
+    extra_bits = 'i686'
+
+extra = None
+
+# Figure out our target triple
+if sys.platform == 'linux' or sys.platform == 'linux2':
+    host = host_bits + '-unknown-linux-gnu'
+    extra = extra_bits + '-unknown-linux-gnu'
+elif sys.platform == 'darwin':
+    host = host_bits + '-apple-darwin'
+    extra = extra_bits + '-apple-darwin'
+elif sys.platform == 'win32':
+    if os.environ.get('MSVC') == '1':
+        host = host_bits + '-pc-windows-msvc'
+    else:
+        host = host_bits + '-pc-windows-gnu'
+else:
+    raise "Unknown platform"
+
+rust_date = open('src/rustversion.txt').read().strip()
+url = 'https://static.rust-lang.org/dist/' + rust_date
+
+def install_via_tarballs():
+    if os.path.isdir("rustc-install"):
+        shutil.rmtree("rustc-install")
+
+    host_fname = 'rustc-nightly-' + host + '.tar.gz'
+    download.get(url + '/' + host_fname, host_fname)
+    download.unpack(host_fname, "rustc-install", quiet=True)
+    os.remove(host_fname)
+
+    if extra != None:
+        extra_fname = 'rustc-nightly-' + extra + '.tar.gz'
+        print("adding target libs for " + extra)
+        download.get(url + '/' + extra_fname, extra_fname)
+        manifest = open("rustc-install/rustc/manifest.in", "a")
+        folder = extra_fname.replace(".tar.gz", "")
+        with contextlib.closing(tarfile.open(extra_fname)) as tar:
+            for p in tar.getnames():
+                if not "rustc/lib/rustlib/" + extra in p:
+                    continue
+                name = p.replace(folder + "/", "", 1)
+                dst = "rustc-install/" + name
+                f = tar.extract(p, "rustc-install")
+                tp = os.path.join("rustc-install", p)
+                if os.path.isdir(tp) and os.path.exists(dst):
+                    continue
+                shutil.move(tp, dst)
+                if not os.path.isdir(dst):
+                    manifest.write(p.replace(folder + "/rustc/", "file:") + "\n")
+        shutil.rmtree("rustc-install/" + folder)
+        os.remove(extra_fname)
+
+    if os.path.isdir("rustc"):
+        shutil.rmtree("rustc")
+    os.rename("rustc-install/rustc", "rustc")
+    shutil.rmtree("rustc-install")
+
+install_via_tarballs()